{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "37b7a56d-f00b-43e7-9716-666530e63f29",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.svm import SVC\n",
    "from sklearn.naive_bayes import MultinomialNB\n",
    "from sklearn.naive_bayes import BernoulliNB\n",
    "from sklearn.metrics import classification_report\n",
    "import nltk\n",
    "import re\n",
    "from nltk.corpus import stopwords\n",
    "from nltk.tokenize import word_tokenize\n",
    "import string\n",
    "import shutil\n",
    "from itertools import islice\n",
    "from scipy.sparse import vstack"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "5fde0607-227f-4d93-86f0-2fbdddb756da",
   "metadata": {},
   "outputs": [],
   "source": [
    "# 下载 NLTK 的停用词和分词工具\n",
    "# nltk.download('punkt_tab')\n",
    "# nltk.download('stopwords')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "60123df2-82ed-4fa7-a453-2c0875730b2d",
   "metadata": {},
   "outputs": [],
   "source": [
    "# 文本预处理函数\n",
    "def preprocess_text(text):\n",
    "    text = re.sub(r'[^\\w\\s\\u4e00-\\u9fa5]', '', text)\n",
    "    # 转为小写\n",
    "    text = text.lower()\n",
    "    # 去除标点符号\n",
    "    text = text.translate(str.maketrans('', '', string.punctuation))\n",
    "    # 分词\n",
    "    tokens = word_tokenize(text)\n",
    "    # 去除停用词\n",
    "    tokens = [word for word in tokens if word not in stopwords.words('english')]\n",
    "    return ' '.join(tokens)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "c8afe63a-f42f-4091-9226-dd9d28487c90",
   "metadata": {},
   "outputs": [],
   "source": [
    "# 读取数据\n",
    "def load_data(file_path):\n",
    "    texts = []\n",
    "    labels = []\n",
    "    \n",
    "    with open(file_path, 'r', encoding='utf-8') as f:\n",
    "        for line in f:\n",
    "            # 分割行，提取标签和文本\n",
    "            parts = line.strip().split(' +++$+++ ')\n",
    "            if len(parts) == 2:\n",
    "                label = int(parts[0])  # 将标签转换为整数\n",
    "                text = parts[1]        # 文本部分\n",
    "                texts.append(preprocess_text(text))  # 预处理文本\n",
    "                labels.append(label)   # 保存标签\n",
    "\n",
    "    return texts, labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "3d2fdc96-b714-492c-909a-d3450d0f1b2b",
   "metadata": {},
   "outputs": [],
   "source": [
    "# 读取无标签数据的函数\n",
    "def load_unlabeled_data(file_path):\n",
    "    texts = []\n",
    "    \n",
    "    # 打开文件并逐行读取\n",
    "    with open(file_path, 'r', encoding='utf-8') as f:\n",
    "        for line in islice(f, 300000):\n",
    "            # 去除行末的换行符和空格\n",
    "            text = line.strip()\n",
    "            if text:  # 确保文本不为空\n",
    "                texts.append(text)  # 将文本添加到列表中\n",
    "\n",
    "    return texts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "a8263bb8-e7ad-4e3d-a224-fdcdb41f2364",
   "metadata": {},
   "outputs": [],
   "source": [
    "# 读取测试数据的函数\n",
    "def load_test_data(file_path):\n",
    "    texts = []\n",
    "    \n",
    "    # 打开文件并逐行读取\n",
    "    with open('data/test.txt', 'r', encoding='utf-8', errors='ignore') as f:\n",
    "        for line in f:\n",
    "            # 去除行末的换行符和空格\n",
    "            line = line.strip()\n",
    "            if line:  # 确保文本不为空\n",
    "                # 分割行，提取文本\n",
    "                _, text = line.split(',', 1)  # 只分割一次，忽略序号\n",
    "                text = re.sub(r'[^\\w\\s\\u4e00-\\u9fa5]', '', text)\n",
    "                texts.append(text.strip())  # 去除文本前后的空白字符\n",
    "\n",
    "    return texts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "f579bade-02a8-4058-8bf6-3005e79410c9",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "load train data………………\n"
     ]
    }
   ],
   "source": [
    "# 加载数据并预处理\n",
    "file_path = 'data/train.txt'  # 替换为你的文件路径\n",
    "print(\"load train data………………\")\n",
    "texts, labels = load_data(file_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "48c2e09a-ab17-46b3-ab93-dcd2041f78a1",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "data columns:  ['text', 'label']\n"
     ]
    }
   ],
   "source": [
    "# 将结果转换为 DataFrame（可选）\n",
    "data = pd.DataFrame({'text': texts, 'label': labels})\n",
    "print(\"data columns: \", data.columns.tolist())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "7ffb4fc3-1fca-4139-ba96-bf6de4c49e23",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "X_train:  100000\n",
      "y_train:  100000\n"
     ]
    }
   ],
   "source": [
    "# 特征提取\n",
    "vectorizer = TfidfVectorizer(max_features=3000)\n",
    "X_train = vectorizer.fit_transform(data['text']).toarray()\n",
    "print(\"X_train: \", len(X_train))\n",
    "y_train = data['label'].values\n",
    "print(\"y_train: \", len(y_train))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "87e52c49-631c-4b84-85d3-1df92680ee2d",
   "metadata": {},
   "outputs": [],
   "source": [
    "# # 分割训练集和验证集\n",
    "# X_train, X_val, y_train, y_val = train_test_split(X_train, y_train, test_size=0.2, random_state=42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "e4833b44-e79d-40cd-bf00-ab7ad117653a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style>#sk-container-id-1 {\n",
       "  /* Definition of color scheme common for light and dark mode */\n",
       "  --sklearn-color-text: #000;\n",
       "  --sklearn-color-text-muted: #666;\n",
       "  --sklearn-color-line: gray;\n",
       "  /* Definition of color scheme for unfitted estimators */\n",
       "  --sklearn-color-unfitted-level-0: #fff5e6;\n",
       "  --sklearn-color-unfitted-level-1: #f6e4d2;\n",
       "  --sklearn-color-unfitted-level-2: #ffe0b3;\n",
       "  --sklearn-color-unfitted-level-3: chocolate;\n",
       "  /* Definition of color scheme for fitted estimators */\n",
       "  --sklearn-color-fitted-level-0: #f0f8ff;\n",
       "  --sklearn-color-fitted-level-1: #d4ebff;\n",
       "  --sklearn-color-fitted-level-2: #b3dbfd;\n",
       "  --sklearn-color-fitted-level-3: cornflowerblue;\n",
       "\n",
       "  /* Specific color for light theme */\n",
       "  --sklearn-color-text-on-default-background: var(--sg-text-color, var(--theme-code-foreground, var(--jp-content-font-color1, black)));\n",
       "  --sklearn-color-background: var(--sg-background-color, var(--theme-background, var(--jp-layout-color0, white)));\n",
       "  --sklearn-color-border-box: var(--sg-text-color, var(--theme-code-foreground, var(--jp-content-font-color1, black)));\n",
       "  --sklearn-color-icon: #696969;\n",
       "\n",
       "  @media (prefers-color-scheme: dark) {\n",
       "    /* Redefinition of color scheme for dark theme */\n",
       "    --sklearn-color-text-on-default-background: var(--sg-text-color, var(--theme-code-foreground, var(--jp-content-font-color1, white)));\n",
       "    --sklearn-color-background: var(--sg-background-color, var(--theme-background, var(--jp-layout-color0, #111)));\n",
       "    --sklearn-color-border-box: var(--sg-text-color, var(--theme-code-foreground, var(--jp-content-font-color1, white)));\n",
       "    --sklearn-color-icon: #878787;\n",
       "  }\n",
       "}\n",
       "\n",
       "#sk-container-id-1 {\n",
       "  color: var(--sklearn-color-text);\n",
       "}\n",
       "\n",
       "#sk-container-id-1 pre {\n",
       "  padding: 0;\n",
       "}\n",
       "\n",
       "#sk-container-id-1 input.sk-hidden--visually {\n",
       "  border: 0;\n",
       "  clip: rect(1px 1px 1px 1px);\n",
       "  clip: rect(1px, 1px, 1px, 1px);\n",
       "  height: 1px;\n",
       "  margin: -1px;\n",
       "  overflow: hidden;\n",
       "  padding: 0;\n",
       "  position: absolute;\n",
       "  width: 1px;\n",
       "}\n",
       "\n",
       "#sk-container-id-1 div.sk-dashed-wrapped {\n",
       "  border: 1px dashed var(--sklearn-color-line);\n",
       "  margin: 0 0.4em 0.5em 0.4em;\n",
       "  box-sizing: border-box;\n",
       "  padding-bottom: 0.4em;\n",
       "  background-color: var(--sklearn-color-background);\n",
       "}\n",
       "\n",
       "#sk-container-id-1 div.sk-container {\n",
       "  /* jupyter's `normalize.less` sets `[hidden] { display: none; }`\n",
       "     but bootstrap.min.css set `[hidden] { display: none !important; }`\n",
       "     so we also need the `!important` here to be able to override the\n",
       "     default hidden behavior on the sphinx rendered scikit-learn.org.\n",
       "     See: https://github.com/scikit-learn/scikit-learn/issues/21755 */\n",
       "  display: inline-block !important;\n",
       "  position: relative;\n",
       "}\n",
       "\n",
       "#sk-container-id-1 div.sk-text-repr-fallback {\n",
       "  display: none;\n",
       "}\n",
       "\n",
       "div.sk-parallel-item,\n",
       "div.sk-serial,\n",
       "div.sk-item {\n",
       "  /* draw centered vertical line to link estimators */\n",
       "  background-image: linear-gradient(var(--sklearn-color-text-on-default-background), var(--sklearn-color-text-on-default-background));\n",
       "  background-size: 2px 100%;\n",
       "  background-repeat: no-repeat;\n",
       "  background-position: center center;\n",
       "}\n",
       "\n",
       "/* Parallel-specific style estimator block */\n",
       "\n",
       "#sk-container-id-1 div.sk-parallel-item::after {\n",
       "  content: \"\";\n",
       "  width: 100%;\n",
       "  border-bottom: 2px solid var(--sklearn-color-text-on-default-background);\n",
       "  flex-grow: 1;\n",
       "}\n",
       "\n",
       "#sk-container-id-1 div.sk-parallel {\n",
       "  display: flex;\n",
       "  align-items: stretch;\n",
       "  justify-content: center;\n",
       "  background-color: var(--sklearn-color-background);\n",
       "  position: relative;\n",
       "}\n",
       "\n",
       "#sk-container-id-1 div.sk-parallel-item {\n",
       "  display: flex;\n",
       "  flex-direction: column;\n",
       "}\n",
       "\n",
       "#sk-container-id-1 div.sk-parallel-item:first-child::after {\n",
       "  align-self: flex-end;\n",
       "  width: 50%;\n",
       "}\n",
       "\n",
       "#sk-container-id-1 div.sk-parallel-item:last-child::after {\n",
       "  align-self: flex-start;\n",
       "  width: 50%;\n",
       "}\n",
       "\n",
       "#sk-container-id-1 div.sk-parallel-item:only-child::after {\n",
       "  width: 0;\n",
       "}\n",
       "\n",
       "/* Serial-specific style estimator block */\n",
       "\n",
       "#sk-container-id-1 div.sk-serial {\n",
       "  display: flex;\n",
       "  flex-direction: column;\n",
       "  align-items: center;\n",
       "  background-color: var(--sklearn-color-background);\n",
       "  padding-right: 1em;\n",
       "  padding-left: 1em;\n",
       "}\n",
       "\n",
       "\n",
       "/* Toggleable style: style used for estimator/Pipeline/ColumnTransformer box that is\n",
       "clickable and can be expanded/collapsed.\n",
       "- Pipeline and ColumnTransformer use this feature and define the default style\n",
       "- Estimators will overwrite some part of the style using the `sk-estimator` class\n",
       "*/\n",
       "\n",
       "/* Pipeline and ColumnTransformer style (default) */\n",
       "\n",
       "#sk-container-id-1 div.sk-toggleable {\n",
       "  /* Default theme specific background. It is overwritten whether we have a\n",
       "  specific estimator or a Pipeline/ColumnTransformer */\n",
       "  background-color: var(--sklearn-color-background);\n",
       "}\n",
       "\n",
       "/* Toggleable label */\n",
       "#sk-container-id-1 label.sk-toggleable__label {\n",
       "  cursor: pointer;\n",
       "  display: flex;\n",
       "  width: 100%;\n",
       "  margin-bottom: 0;\n",
       "  padding: 0.5em;\n",
       "  box-sizing: border-box;\n",
       "  text-align: center;\n",
       "  align-items: start;\n",
       "  justify-content: space-between;\n",
       "  gap: 0.5em;\n",
       "}\n",
       "\n",
       "#sk-container-id-1 label.sk-toggleable__label .caption {\n",
       "  font-size: 0.6rem;\n",
       "  font-weight: lighter;\n",
       "  color: var(--sklearn-color-text-muted);\n",
       "}\n",
       "\n",
       "#sk-container-id-1 label.sk-toggleable__label-arrow:before {\n",
       "  /* Arrow on the left of the label */\n",
       "  content: \"▸\";\n",
       "  float: left;\n",
       "  margin-right: 0.25em;\n",
       "  color: var(--sklearn-color-icon);\n",
       "}\n",
       "\n",
       "#sk-container-id-1 label.sk-toggleable__label-arrow:hover:before {\n",
       "  color: var(--sklearn-color-text);\n",
       "}\n",
       "\n",
       "/* Toggleable content - dropdown */\n",
       "\n",
       "#sk-container-id-1 div.sk-toggleable__content {\n",
       "  max-height: 0;\n",
       "  max-width: 0;\n",
       "  overflow: hidden;\n",
       "  text-align: left;\n",
       "  /* unfitted */\n",
       "  background-color: var(--sklearn-color-unfitted-level-0);\n",
       "}\n",
       "\n",
       "#sk-container-id-1 div.sk-toggleable__content.fitted {\n",
       "  /* fitted */\n",
       "  background-color: var(--sklearn-color-fitted-level-0);\n",
       "}\n",
       "\n",
       "#sk-container-id-1 div.sk-toggleable__content pre {\n",
       "  margin: 0.2em;\n",
       "  border-radius: 0.25em;\n",
       "  color: var(--sklearn-color-text);\n",
       "  /* unfitted */\n",
       "  background-color: var(--sklearn-color-unfitted-level-0);\n",
       "}\n",
       "\n",
       "#sk-container-id-1 div.sk-toggleable__content.fitted pre {\n",
       "  /* unfitted */\n",
       "  background-color: var(--sklearn-color-fitted-level-0);\n",
       "}\n",
       "\n",
       "#sk-container-id-1 input.sk-toggleable__control:checked~div.sk-toggleable__content {\n",
       "  /* Expand drop-down */\n",
       "  max-height: 200px;\n",
       "  max-width: 100%;\n",
       "  overflow: auto;\n",
       "}\n",
       "\n",
       "#sk-container-id-1 input.sk-toggleable__control:checked~label.sk-toggleable__label-arrow:before {\n",
       "  content: \"▾\";\n",
       "}\n",
       "\n",
       "/* Pipeline/ColumnTransformer-specific style */\n",
       "\n",
       "#sk-container-id-1 div.sk-label input.sk-toggleable__control:checked~label.sk-toggleable__label {\n",
       "  color: var(--sklearn-color-text);\n",
       "  background-color: var(--sklearn-color-unfitted-level-2);\n",
       "}\n",
       "\n",
       "#sk-container-id-1 div.sk-label.fitted input.sk-toggleable__control:checked~label.sk-toggleable__label {\n",
       "  background-color: var(--sklearn-color-fitted-level-2);\n",
       "}\n",
       "\n",
       "/* Estimator-specific style */\n",
       "\n",
       "/* Colorize estimator box */\n",
       "#sk-container-id-1 div.sk-estimator input.sk-toggleable__control:checked~label.sk-toggleable__label {\n",
       "  /* unfitted */\n",
       "  background-color: var(--sklearn-color-unfitted-level-2);\n",
       "}\n",
       "\n",
       "#sk-container-id-1 div.sk-estimator.fitted input.sk-toggleable__control:checked~label.sk-toggleable__label {\n",
       "  /* fitted */\n",
       "  background-color: var(--sklearn-color-fitted-level-2);\n",
       "}\n",
       "\n",
       "#sk-container-id-1 div.sk-label label.sk-toggleable__label,\n",
       "#sk-container-id-1 div.sk-label label {\n",
       "  /* The background is the default theme color */\n",
       "  color: var(--sklearn-color-text-on-default-background);\n",
       "}\n",
       "\n",
       "/* On hover, darken the color of the background */\n",
       "#sk-container-id-1 div.sk-label:hover label.sk-toggleable__label {\n",
       "  color: var(--sklearn-color-text);\n",
       "  background-color: var(--sklearn-color-unfitted-level-2);\n",
       "}\n",
       "\n",
       "/* Label box, darken color on hover, fitted */\n",
       "#sk-container-id-1 div.sk-label.fitted:hover label.sk-toggleable__label.fitted {\n",
       "  color: var(--sklearn-color-text);\n",
       "  background-color: var(--sklearn-color-fitted-level-2);\n",
       "}\n",
       "\n",
       "/* Estimator label */\n",
       "\n",
       "#sk-container-id-1 div.sk-label label {\n",
       "  font-family: monospace;\n",
       "  font-weight: bold;\n",
       "  display: inline-block;\n",
       "  line-height: 1.2em;\n",
       "}\n",
       "\n",
       "#sk-container-id-1 div.sk-label-container {\n",
       "  text-align: center;\n",
       "}\n",
       "\n",
       "/* Estimator-specific */\n",
       "#sk-container-id-1 div.sk-estimator {\n",
       "  font-family: monospace;\n",
       "  border: 1px dotted var(--sklearn-color-border-box);\n",
       "  border-radius: 0.25em;\n",
       "  box-sizing: border-box;\n",
       "  margin-bottom: 0.5em;\n",
       "  /* unfitted */\n",
       "  background-color: var(--sklearn-color-unfitted-level-0);\n",
       "}\n",
       "\n",
       "#sk-container-id-1 div.sk-estimator.fitted {\n",
       "  /* fitted */\n",
       "  background-color: var(--sklearn-color-fitted-level-0);\n",
       "}\n",
       "\n",
       "/* on hover */\n",
       "#sk-container-id-1 div.sk-estimator:hover {\n",
       "  /* unfitted */\n",
       "  background-color: var(--sklearn-color-unfitted-level-2);\n",
       "}\n",
       "\n",
       "#sk-container-id-1 div.sk-estimator.fitted:hover {\n",
       "  /* fitted */\n",
       "  background-color: var(--sklearn-color-fitted-level-2);\n",
       "}\n",
       "\n",
       "/* Specification for estimator info (e.g. \"i\" and \"?\") */\n",
       "\n",
       "/* Common style for \"i\" and \"?\" */\n",
       "\n",
       ".sk-estimator-doc-link,\n",
       "a:link.sk-estimator-doc-link,\n",
       "a:visited.sk-estimator-doc-link {\n",
       "  float: right;\n",
       "  font-size: smaller;\n",
       "  line-height: 1em;\n",
       "  font-family: monospace;\n",
       "  background-color: var(--sklearn-color-background);\n",
       "  border-radius: 1em;\n",
       "  height: 1em;\n",
       "  width: 1em;\n",
       "  text-decoration: none !important;\n",
       "  margin-left: 0.5em;\n",
       "  text-align: center;\n",
       "  /* unfitted */\n",
       "  border: var(--sklearn-color-unfitted-level-1) 1pt solid;\n",
       "  color: var(--sklearn-color-unfitted-level-1);\n",
       "}\n",
       "\n",
       ".sk-estimator-doc-link.fitted,\n",
       "a:link.sk-estimator-doc-link.fitted,\n",
       "a:visited.sk-estimator-doc-link.fitted {\n",
       "  /* fitted */\n",
       "  border: var(--sklearn-color-fitted-level-1) 1pt solid;\n",
       "  color: var(--sklearn-color-fitted-level-1);\n",
       "}\n",
       "\n",
       "/* On hover */\n",
       "div.sk-estimator:hover .sk-estimator-doc-link:hover,\n",
       ".sk-estimator-doc-link:hover,\n",
       "div.sk-label-container:hover .sk-estimator-doc-link:hover,\n",
       ".sk-estimator-doc-link:hover {\n",
       "  /* unfitted */\n",
       "  background-color: var(--sklearn-color-unfitted-level-3);\n",
       "  color: var(--sklearn-color-background);\n",
       "  text-decoration: none;\n",
       "}\n",
       "\n",
       "div.sk-estimator.fitted:hover .sk-estimator-doc-link.fitted:hover,\n",
       ".sk-estimator-doc-link.fitted:hover,\n",
       "div.sk-label-container:hover .sk-estimator-doc-link.fitted:hover,\n",
       ".sk-estimator-doc-link.fitted:hover {\n",
       "  /* fitted */\n",
       "  background-color: var(--sklearn-color-fitted-level-3);\n",
       "  color: var(--sklearn-color-background);\n",
       "  text-decoration: none;\n",
       "}\n",
       "\n",
       "/* Span, style for the box shown on hovering the info icon */\n",
       ".sk-estimator-doc-link span {\n",
       "  display: none;\n",
       "  z-index: 9999;\n",
       "  position: relative;\n",
       "  font-weight: normal;\n",
       "  right: .2ex;\n",
       "  padding: .5ex;\n",
       "  margin: .5ex;\n",
       "  width: min-content;\n",
       "  min-width: 20ex;\n",
       "  max-width: 50ex;\n",
       "  color: var(--sklearn-color-text);\n",
       "  box-shadow: 2pt 2pt 4pt #999;\n",
       "  /* unfitted */\n",
       "  background: var(--sklearn-color-unfitted-level-0);\n",
       "  border: .5pt solid var(--sklearn-color-unfitted-level-3);\n",
       "}\n",
       "\n",
       ".sk-estimator-doc-link.fitted span {\n",
       "  /* fitted */\n",
       "  background: var(--sklearn-color-fitted-level-0);\n",
       "  border: var(--sklearn-color-fitted-level-3);\n",
       "}\n",
       "\n",
       ".sk-estimator-doc-link:hover span {\n",
       "  display: block;\n",
       "}\n",
       "\n",
       "/* \"?\"-specific style due to the `<a>` HTML tag */\n",
       "\n",
       "#sk-container-id-1 a.estimator_doc_link {\n",
       "  float: right;\n",
       "  font-size: 1rem;\n",
       "  line-height: 1em;\n",
       "  font-family: monospace;\n",
       "  background-color: var(--sklearn-color-background);\n",
       "  border-radius: 1rem;\n",
       "  height: 1rem;\n",
       "  width: 1rem;\n",
       "  text-decoration: none;\n",
       "  /* unfitted */\n",
       "  color: var(--sklearn-color-unfitted-level-1);\n",
       "  border: var(--sklearn-color-unfitted-level-1) 1pt solid;\n",
       "}\n",
       "\n",
       "#sk-container-id-1 a.estimator_doc_link.fitted {\n",
       "  /* fitted */\n",
       "  border: var(--sklearn-color-fitted-level-1) 1pt solid;\n",
       "  color: var(--sklearn-color-fitted-level-1);\n",
       "}\n",
       "\n",
       "/* On hover */\n",
       "#sk-container-id-1 a.estimator_doc_link:hover {\n",
       "  /* unfitted */\n",
       "  background-color: var(--sklearn-color-unfitted-level-3);\n",
       "  color: var(--sklearn-color-background);\n",
       "  text-decoration: none;\n",
       "}\n",
       "\n",
       "#sk-container-id-1 a.estimator_doc_link.fitted:hover {\n",
       "  /* fitted */\n",
       "  background-color: var(--sklearn-color-fitted-level-3);\n",
       "}\n",
       "</style><div id=\"sk-container-id-1\" class=\"sk-top-container\"><div class=\"sk-text-repr-fallback\"><pre>LogisticRegression()</pre><b>In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook. <br />On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.</b></div><div class=\"sk-container\" hidden><div class=\"sk-item\"><div class=\"sk-estimator fitted sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-1\" type=\"checkbox\" checked><label for=\"sk-estimator-id-1\" class=\"sk-toggleable__label fitted sk-toggleable__label-arrow\"><div><div>LogisticRegression</div></div><div><a class=\"sk-estimator-doc-link fitted\" rel=\"noreferrer\" target=\"_blank\" href=\"https://scikit-learn.org/1.6/modules/generated/sklearn.linear_model.LogisticRegression.html\">?<span>Documentation for LogisticRegression</span></a><span class=\"sk-estimator-doc-link fitted\">i<span>Fitted</span></span></div></label><div class=\"sk-toggleable__content fitted\"><pre>LogisticRegression()</pre></div> </div></div></div></div>"
      ],
      "text/plain": [
       "LogisticRegression()"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 构建逻辑回归模型并训练\n",
    "model = LogisticRegression()\n",
    "# 构建多项式朴素贝叶斯模型并训练\n",
    "# model = MultinomialNB()\n",
    "# 构建伯努利朴素贝叶斯模型并训练\n",
    "# model = BernoulliNB()\n",
    "\n",
    "model.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "1601dbe7-42f7-410a-8ca3-af676ba1043b",
   "metadata": {},
   "outputs": [],
   "source": [
    "# # 在验证集上评估模型性能\n",
    "# y_pred_val = model.predict(X_val)\n",
    "# print(\"Validation Classification Report:\")\n",
    "# print(classification_report(y_val, y_pred_val))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "7ed7e3ea-0273-444a-911b-6b9c98adbc11",
   "metadata": {},
   "outputs": [],
   "source": [
    "# 读取无标签数据并进行预处理\n",
    "unlabeled_data = load_unlabeled_data('data/nolabel.txt')  # 确保文件路径正确\n",
    "unlabeled_data = pd.DataFrame({'text': unlabeled_data})\n",
    "# unlabeled_data = unlabeled_data.apply(preprocess_text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "6d9dc59c-96ee-4e07-a649-23dfeefd0520",
   "metadata": {},
   "outputs": [],
   "source": [
    "# unlabeled_data['text'] = unlabeled_data['text'].apply(preprocess_text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "96cc565d-e20d-4eda-91dd-103deefe0974",
   "metadata": {},
   "outputs": [],
   "source": [
    "X_unlabeled = vectorizer.transform(unlabeled_data['text']).toarray()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "f6000896-6403-40a6-83a4-48e3383c6884",
   "metadata": {},
   "outputs": [],
   "source": [
    "# 使用已训练的模型进行预测概率\n",
    "predicted_probs = model.predict_proba(X_unlabeled)  # 获取每个类的预测概率"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "41bcd051-3c17-46d6-8553-4cdf4441fa06",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "new_train_texts:  114337\n",
      "new_train_labels:  114337\n"
     ]
    }
   ],
   "source": [
    "# 设定置信度阈值\n",
    "confidence_threshold = 0.8\n",
    "\n",
    "# 筛选高置信度样本的索引\n",
    "high_confidence_indices = np.where(np.max(predicted_probs, axis=1) > confidence_threshold)[0]\n",
    "high_confidence_samples = unlabeled_data.iloc[high_confidence_indices]\n",
    "high_confidence_labels = np.argmax(predicted_probs[high_confidence_indices], axis=1)  # 获取预测标签\n",
    "\n",
    "# 将高置信度样本添加到训练集中\n",
    "# new_train_texts = data['text'].tolist() + high_confidence_samples['text'].tolist()\n",
    "# new_train_labels = y_train.tolist() + high_confidence_labels.tolist()\n",
    "new_train_texts = high_confidence_samples['text'].tolist()\n",
    "new_train_labels = high_confidence_labels.tolist()\n",
    "print(\"new_train_texts: \", len(new_train_texts))\n",
    "print(\"new_train_labels: \", len(new_train_labels))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "55d538e5-1169-447e-9eea-bab284b0d1ca",
   "metadata": {},
   "outputs": [],
   "source": [
    "# 重新编码新训练集并创建新的 Dataset 对象\n",
    "new_train_encodings = vectorizer.fit_transform(new_train_texts).toarray()  # 更新特征提取器以适应新数据"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "168c9138-d8e3-469a-bd49-55fbba7df281",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished updating the training set with high-confidence samples.\n"
     ]
    }
   ],
   "source": [
    "# 使用新的训练集重新训练模型（可选）\n",
    "model.fit(new_train_encodings, new_train_labels)\n",
    "\n",
    "print(\"Finished updating the training set with high-confidence samples.\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "0f890c71-a9c5-4ec0-a9d2-b6a8d7afe66d",
   "metadata": {},
   "outputs": [],
   "source": [
    "# 测试集上进行推理\n",
    "test_data = load_test_data('data/test.txt')\n",
    "\n",
    "# 将结果转换为 DataFrame（可选）\n",
    "test_data = pd.DataFrame({'text': test_data})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "58469c4a-8b46-4c95-85a4-741aa59c8119",
   "metadata": {},
   "outputs": [],
   "source": [
    "batch_size = 10000\n",
    "test_data_size = len(test_data)\n",
    "num_batches = (test_data_size + batch_size - 1) // batch_size\n",
    "\n",
    "test_predictions = np.concatenate([\n",
    "    model.predict(vectorizer.transform(test_data['text'].iloc[i * batch_size: min((i + 1) * batch_size, test_data_size)]).toarray())\n",
    "    for i in range(num_batches)\n",
    "])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "d6daea6c-cc14-4f5f-89a0-1a6c87cdd8a4",
   "metadata": {},
   "outputs": [],
   "source": [
    "# test_data['text'] = test_data['text'].apply(preprocess_text)\n",
    "# X_test = vectorizer.transform(test_data['text']).toarray()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "81451b53-04ca-4624-b775-302306683498",
   "metadata": {},
   "outputs": [],
   "source": [
    "# 进行预测\n",
    "# test_predictions = model.predict(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "e9b29fb5-240c-49b6-96a7-e30479ceb612",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'data/2025010301/text_sentiment_analysis.ipynb'"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 批次，如果进行了大的更新，建议保存相应代码及预测结果\n",
    "# 批次规则：YYYYMMDDXX，XX每日从01开始\n",
    "path = 'data/2025010301/'\n",
    "if not os.path.exists(path):\n",
    "    os.mkdir(path)\n",
    "# 输出结果或保存到文件中\n",
    "test_data.index.name = 'index'\n",
    "test_data['label'] = test_predictions\n",
    "test_data.to_csv(path+'predictions.csv', index=False, encoding='utf-8')\n",
    "test_data['label'].to_csv(path+'submission.csv', index=True, encoding='utf-8')\n",
    "\n",
    "# Save the notebook files\n",
    "shutil.copy('text_sentiment_analysis.ipynb', path+'text_sentiment_analysis.ipynb')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "87990ea8-a792-47d6-90e3-a90767f2391d",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
